#------------------------------------------------------------------------------
# combine aggregated data
#==============================================================================
load_library = c('bit64','data.table','fst','future.apply','stringr','logger','vroom')
invisible(lapply(load_library, function(x) library(x, character.only=TRUE, quietly= TRUE)))

bucket = '/N/project/iuni_doctorshopping/'

extract_date_from_filename = function(outfile){
  date = str_extract(outfile,"\\d{4}-\\d{2}-\\d{2}")
  return(date)
}

extract_week_from_filename = function(outfile){
  date = str_extract(outfile,"\\d{4}W\\d{2}")
  date = gsub(' ','_',date)
  return(date)
}

# ----------
logger::log_info('now reading data ... state')
all_files = dir(file.path(bucket,'projects','covid_opioid','weekly_division'), full.names=TRUE)

geo_combined = rbindlist(lapply(all_files, function(ff) {
	dt = fread(ff)
	date = extract_week_from_filename(ff)
	dt$date = date
	return(dt)
}))

outfile = file.path(bucket,'projects','covid_opioid','data',
	'processed_data','ses_division_weekly_covid_0101_0930.fst')

logger::log_info('now writing data ... state')
write_fst(geo_combined, outfile, 100)
